package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.net.withub.dataCollector.common.model.TCollectorConfig;
import cn.net.withub.dataCollector.common.model.TCollectorData;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import java.io.IOException;
import java.text.SimpleDateFormat;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

/**
 * 新浪网国内新闻
 */
public class CollectorUtilsXhw extends CrawlerBase {


    public CollectorUtilsXhw(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse, wac, config);
        webAppliction = wac;
        if(webAppliction !=null) collectorService = (CollectorService) webAppliction.getBean("collectorService");
        if(config != null) tCollectorConfig = config;
        sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    }
    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.url();
        String html = page.html();
        String contentType = page.contentType();
        String nextUrl = "";
        String title = "", fbsj = "", subTitle = "", xxly = "";//信息来源
        log.info("--------------------------URL:" + url);
        try{
            if ("http://www.xinhuanet.com/".equals(url)) {//新华网
                Elements elements = page.select("#hpart2L a");
                for (Element e : elements) {
                    nextUrl = e.attr("abs:href");
                    subTitle = e.text();
                    log.info("nextUrl...."+nextUrl);
                    //crawlDatums.add(nextUrl, "newsDetail");
                    TCollectorData tCollectorData = (TCollectorData) collectorService.load(TCollectorData.class, "title like'%" + subTitle + "%'");//%前不能有空格
                    if(tCollectorData==null){
                        crawlDatums.add(new CrawlDatum(nextUrl, "newsDetail").meta("preUrL", url));
                    }
                }
            } else if (page.matchType("newsDetail") && page.matchUrl("http://www.xinhuanet.com.*")) {//单条新闻详情页面
                Elements contentEle = null;
                title = page.select(".h-p3 .h-news .h-title").text();
                fbsj = page.select(".h-p3 .h-news  .h-time").text();
                //不是一个模板...
                xxly = page.select("#source").text();
                if ("".equals(xxly)){
                    xxly = page.select(".h-p3 .h-news  .sub-src .aticle-src").text();
                }
                if ("".equals(xxly)) {
                    Elements e = page.select(".h-p3 .h-news  .h-info span");
                    if (e.size() > 1) {
                        xxly = e.get(1).text();
                        xxly = xxly.substring(xxly.indexOf(":") + 4);
                    }
                }
                contentEle = page.select("#p-detail");
                Elements imgs = contentEle.select("img[src]");
                String id = new CreateNewKey().createId();
                //遍历所有html中所有图片
                for (Element img : imgs) {
                    String src = img.attr("abs:src");
                    //这个图标貌似每个页面都有,无资源的
                    if ("http://www.xinhuanet.com/images/syicon/space.gif".equals(src)) continue;
                    //此处图片地址包含中文会报错
                    //src = fmtUrl(src);
                    log.info("img url:\n" + src);
                    String imageName = src.substring(src.lastIndexOf("/") + 1, src.length());
                    String serverFilePath = "";
                    try {
                        //serverFilePath = HttpClientUtils.getImage(src);
                        serverFilePath = FtpUtil.getImage(src, "");
                        //img.attr("src", serverFilePath);
                        saveElement(id, imageName, src, serverFilePath);
                    } catch (IOException e) {
                        log.error("出错src:" + src + ";父级路径:" + url);
                        e.printStackTrace();
                    }
                }
                contentEle.select("div[class='zan-wap']").remove();
                contentEle.select("div[class='p-tags']").remove();
                contentEle.select("div[class='lb']").remove();
                //String content = contentEle.outerHtml();
                String content = contentEle.html();
                //title = getStr(page.meta("title"));
                String preUrL = getStr(page.meta("preUrL"));
                String text = HTMLSpirit.getTextFromTHML(content);
                int zt = 1;
                try {
                    //保存TCollectorData
                    saveData(id, title, fbsj, xxly, "html", url, "", "", "", "");
                    //保存TCollectorContent
                    saveContent(id,content,text);
                    //保存最大发布时间
                    saveMaxFbsj(fbsj);
                    zt = 1;
                } catch (Exception e) {
                    zt = 0;
                    e.printStackTrace();
                } finally {
                    saveLog(title,fbsj,preUrL,url,zt);
                }
            }

        }catch (Exception e){
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {

    }

}
